{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1dfbb7cb-5fcb-47b0-a25d-c5c26b79aa98",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from fastembed import TextEmbedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c840f70d-a19b-4d53-8237-1a448cf0b361",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedder = TextEmbedding(model_name='jinaai/jina-embeddings-v2-small-en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0668746-7383-4aff-bdca-3b728c8b7d52",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1fcab59b-aac5-4037-b443-a58eac96245e",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = 'I just discovered the course. Can I join now?'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "64ddf2c9-9531-4065-b2a4-b214476cf1ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "q, = list(embedder.embed(query))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "6b775eb1-2f45-4af0-8a95-35b90a690ecd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(-0.11726373885183883)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b991b252-9c82-4508-a8fd-efa0c3fdf077",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5c2d4b6a-7e0a-4b80-a2d0-85796518e061",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(1.0)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "4099f428-c672-4b21-b7b8-ffb1b0509448",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(1.0000000000000002)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.dot(q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94241e6d-546f-427f-9bf0-2cbe8695cf3b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "23b9851b-da32-41a2-8dcb-f3e7596bf9ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = 'Can I still join the course after the start date?'\n",
    "d, = list(embedder.embed(doc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "30b83be5-d502-48da-84b1-10804720d2d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.9008528895674547)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q.dot(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "840abd1b-a9eb-4bb6-8ba2-cb1693d5b6aa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "db3558cf-39d3-4cd1-92ff-15fd0e18a472",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = [{'text': \"Yes, even if you don't register, you're still eligible to submit the homeworks.\\nBe aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute.\",\n",
    "  'section': 'General course-related questions',\n",
    "  'question': 'Course - Can I still join the course after the start date?',\n",
    "  'course': 'data-engineering-zoomcamp'},\n",
    " {'text': 'Yes, we will keep all the materials after the course finishes, so you can follow the course at your own pace after it finishes.\\nYou can also continue looking at the homeworks and continue preparing for the next cohort. I guess you can also start working on your final capstone project.',\n",
    "  'section': 'General course-related questions',\n",
    "  'question': 'Course - Can I follow the course after it finishes?',\n",
    "  'course': 'data-engineering-zoomcamp'},\n",
    " {'text': \"The purpose of this document is to capture frequently asked technical questions\\nThe exact day and hour of the course will be 15th Jan 2024 at 17h00. The course will start with the first  “Office Hours'' live.1\\nSubscribe to course public Google Calendar (it works from Desktop only).\\nRegister before the course starts using this link.\\nJoin the course Telegram channel with announcements.\\nDon’t forget to register in DataTalks.Club's Slack and join the channel.\",\n",
    "  'section': 'General course-related questions',\n",
    "  'question': 'Course - When will the course start?',\n",
    "  'course': 'data-engineering-zoomcamp'},\n",
    " {'text': 'You can start by installing and setting up all the dependencies and requirements:\\nGoogle cloud account\\nGoogle Cloud SDK\\nPython 3 (installed with Anaconda)\\nTerraform\\nGit\\nLook over the prerequisites and syllabus to see if you are comfortable with these subjects.',\n",
    "  'section': 'General course-related questions',\n",
    "  'question': 'Course - What can I do before the course starts?',\n",
    "  'course': 'data-engineering-zoomcamp'},\n",
    " {'text': 'Star the repo! Share it with friends if you find it useful ❣️\\nCreate a PR if you see you can improve the text or the structure of the repository.',\n",
    "  'section': 'General course-related questions',\n",
    "  'question': 'How can we contribute to the course?',\n",
    "  'course': 'data-engineering-zoomcamp'}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3fa0df9a-1718-4c7f-aee5-b879ee0d9d4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "43192199-376b-4325-8535-f89c21f487cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>section</th>\n",
       "      <th>question</th>\n",
       "      <th>course</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Yes, even if you don't register, you're still ...</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - Can I still join the course after the...</td>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Yes, we will keep all the materials after the ...</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - Can I follow the course after it fini...</td>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - When will the course start?</td>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>Course - What can I do before the course starts?</td>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Star the repo! Share it with friends if you fi...</td>\n",
       "      <td>General course-related questions</td>\n",
       "      <td>How can we contribute to the course?</td>\n",
       "      <td>data-engineering-zoomcamp</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  Yes, even if you don't register, you're still ...   \n",
       "1  Yes, we will keep all the materials after the ...   \n",
       "2  The purpose of this document is to capture fre...   \n",
       "3  You can start by installing and setting up all...   \n",
       "4  Star the repo! Share it with friends if you fi...   \n",
       "\n",
       "                            section  \\\n",
       "0  General course-related questions   \n",
       "1  General course-related questions   \n",
       "2  General course-related questions   \n",
       "3  General course-related questions   \n",
       "4  General course-related questions   \n",
       "\n",
       "                                            question  \\\n",
       "0  Course - Can I still join the course after the...   \n",
       "1  Course - Can I follow the course after it fini...   \n",
       "2               Course - When will the course start?   \n",
       "3   Course - What can I do before the course starts?   \n",
       "4               How can we contribute to the course?   \n",
       "\n",
       "                      course  \n",
       "0  data-engineering-zoomcamp  \n",
       "1  data-engineering-zoomcamp  \n",
       "2  data-engineering-zoomcamp  \n",
       "3  data-engineering-zoomcamp  \n",
       "4  data-engineering-zoomcamp  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(documents)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "edc54e47-441d-40cb-8211-1eadb8a0a14b",
   "metadata": {},
   "outputs": [],
   "source": [
    "V_text = list(embedder.embed(df.text))\n",
    "V_text = np.array(V_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "fe405bcd-1a14-4aea-9629-84a404437ce4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.76296845, 0.81823782, 0.80853974, 0.71330788, 0.73044992])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity = V_text.dot(q)\n",
    "similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5ea4341f-a35e-4651-b6ef-dbeba90704bf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.int64(1)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity.argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3583114f-d9a7-472a-a192-5e61ca60d4d7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b8d96e62-2142-464a-baaa-c4e078e42f9f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.int64(0)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "V_text = list(embedder.embed(df.question + ' ' + df.text))\n",
    "V_text = np.array(V_text)\n",
    "\n",
    "similarity = V_text.dot(q)\n",
    "similarity.argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82b76761-bf71-464c-bea5-2f1d3a9702ac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "6369b381-d71d-454e-aa75-ffa537282eb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests \n",
    "\n",
    "docs_url = 'https://github.com/alexeygrigorev/llm-rag-workshop/raw/main/notebooks/documents.json'\n",
    "docs_response = requests.get(docs_url)\n",
    "documents_raw = docs_response.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "0771f592-5081-4d52-884f-df49d9675515",
   "metadata": {},
   "outputs": [],
   "source": [
    "documents = []\n",
    "\n",
    "for course in documents_raw:\n",
    "    course_name = course['course']\n",
    "    if course_name != 'machine-learning-zoomcamp':\n",
    "        continue\n",
    "\n",
    "    for doc in course['documents']:\n",
    "        doc['course'] = course_name\n",
    "        documents.append(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "720108a3-5da4-47e4-9ea2-1b44f154fb59",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "ae81e636-6007-4fa1-980d-cb3df6d87698",
   "metadata": {},
   "outputs": [],
   "source": [
    "models = TextEmbedding.list_supported_models()\n",
    "df_models = pd.DataFrame(models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "ea83664e-f193-48cf-a846-f5d62f3e4491",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>sources</th>\n",
       "      <th>model_file</th>\n",
       "      <th>description</th>\n",
       "      <th>license</th>\n",
       "      <th>size_in_GB</th>\n",
       "      <th>additional_files</th>\n",
       "      <th>dim</th>\n",
       "      <th>tasks</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>BAAI/bge-small-en</td>\n",
       "      <td>{'hf': 'Qdrant/bge-small-en', 'url': 'https://...</td>\n",
       "      <td>model_optimized.onnx</td>\n",
       "      <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
       "      <td>mit</td>\n",
       "      <td>0.130</td>\n",
       "      <td>[]</td>\n",
       "      <td>384</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>BAAI/bge-small-en-v1.5</td>\n",
       "      <td>{'hf': 'qdrant/bge-small-en-v1.5-onnx-q', 'url...</td>\n",
       "      <td>model_optimized.onnx</td>\n",
       "      <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
       "      <td>mit</td>\n",
       "      <td>0.067</td>\n",
       "      <td>[]</td>\n",
       "      <td>384</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>snowflake/snowflake-arctic-embed-xs</td>\n",
       "      <td>{'hf': 'snowflake/snowflake-arctic-embed-xs', ...</td>\n",
       "      <td>onnx/model.onnx</td>\n",
       "      <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
       "      <td>apache-2.0</td>\n",
       "      <td>0.090</td>\n",
       "      <td>[]</td>\n",
       "      <td>384</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>snowflake/snowflake-arctic-embed-s</td>\n",
       "      <td>{'hf': 'snowflake/snowflake-arctic-embed-s', '...</td>\n",
       "      <td>onnx/model.onnx</td>\n",
       "      <td>Text embeddings, Unimodal (text), English, 512...</td>\n",
       "      <td>apache-2.0</td>\n",
       "      <td>0.130</td>\n",
       "      <td>[]</td>\n",
       "      <td>384</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>sentence-transformers/all-MiniLM-L6-v2</td>\n",
       "      <td>{'hf': 'qdrant/all-MiniLM-L6-v2-onnx', 'url': ...</td>\n",
       "      <td>model.onnx</td>\n",
       "      <td>Text embeddings, Unimodal (text), English, 256...</td>\n",
       "      <td>apache-2.0</td>\n",
       "      <td>0.090</td>\n",
       "      <td>[]</td>\n",
       "      <td>384</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>sentence-transformers/paraphrase-multilingual-...</td>\n",
       "      <td>{'hf': 'qdrant/paraphrase-multilingual-MiniLM-...</td>\n",
       "      <td>model_optimized.onnx</td>\n",
       "      <td>Text embeddings, Unimodal (text), Multilingual...</td>\n",
       "      <td>apache-2.0</td>\n",
       "      <td>0.220</td>\n",
       "      <td>[]</td>\n",
       "      <td>384</td>\n",
       "      <td>{}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                model  \\\n",
       "3                                   BAAI/bge-small-en   \n",
       "4                              BAAI/bge-small-en-v1.5   \n",
       "7                 snowflake/snowflake-arctic-embed-xs   \n",
       "8                  snowflake/snowflake-arctic-embed-s   \n",
       "14             sentence-transformers/all-MiniLM-L6-v2   \n",
       "26  sentence-transformers/paraphrase-multilingual-...   \n",
       "\n",
       "                                              sources            model_file  \\\n",
       "3   {'hf': 'Qdrant/bge-small-en', 'url': 'https://...  model_optimized.onnx   \n",
       "4   {'hf': 'qdrant/bge-small-en-v1.5-onnx-q', 'url...  model_optimized.onnx   \n",
       "7   {'hf': 'snowflake/snowflake-arctic-embed-xs', ...       onnx/model.onnx   \n",
       "8   {'hf': 'snowflake/snowflake-arctic-embed-s', '...       onnx/model.onnx   \n",
       "14  {'hf': 'qdrant/all-MiniLM-L6-v2-onnx', 'url': ...            model.onnx   \n",
       "26  {'hf': 'qdrant/paraphrase-multilingual-MiniLM-...  model_optimized.onnx   \n",
       "\n",
       "                                          description     license  size_in_GB  \\\n",
       "3   Text embeddings, Unimodal (text), English, 512...         mit       0.130   \n",
       "4   Text embeddings, Unimodal (text), English, 512...         mit       0.067   \n",
       "7   Text embeddings, Unimodal (text), English, 512...  apache-2.0       0.090   \n",
       "8   Text embeddings, Unimodal (text), English, 512...  apache-2.0       0.130   \n",
       "14  Text embeddings, Unimodal (text), English, 256...  apache-2.0       0.090   \n",
       "26  Text embeddings, Unimodal (text), Multilingual...  apache-2.0       0.220   \n",
       "\n",
       "   additional_files  dim tasks  \n",
       "3                []  384    {}  \n",
       "4                []  384    {}  \n",
       "7                []  384    {}  \n",
       "8                []  384    {}  \n",
       "14               []  384    {}  \n",
       "26               []  384    {}  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_models[df_models.dim == df_models.dim.min()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b9f37b3-3da0-46c2-bda3-299c0704f84e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "7fec91cd-efe5-4114-b1d5-e13b2fd02ef8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from qdrant_client import QdrantClient, models\n",
    "qd_client = QdrantClient(\"http://localhost:6333\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "82addd1e-3502-4a2f-b441-af812d5f33d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_DIMENSIONALITY = 384\n",
    "model_handle = \"BAAI/bge-small-en\"\n",
    "collection_name = \"llmzoomcamp-homework\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "3c8f1818-7189-4966-9a76-253141797d7a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qd_client.delete_collection(collection_name=collection_name)\n",
    "\n",
    "qd_client.create_collection(\n",
    "    collection_name=collection_name,\n",
    "    vectors_config=models.VectorParams(\n",
    "        size=EMBEDDING_DIMENSIONALITY,\n",
    "        distance=models.Distance.COSINE\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "c3d1d5c0-7477-4419-b483-67f6334a05dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e26c33416d1647428cc2470c56d8f7e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 5 files:   0%|          | 0/5 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cd643d43865445c83c86796928ce2d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model_optimized.onnx:   0%|          | 0.00/133M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e873da81fbd043d8b35606f4dc8ec069",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/701 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\alexe\\miniconda3\\Lib\\site-packages\\huggingface_hub\\file_download.py:157: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\alexe\\AppData\\Local\\Temp\\fastembed_cache\\models--Qdrant--bge-small-en. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d11fbbe65d794fbf807f71400ff23ba3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/366 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a68ba39aed7477e9df68de2c1abf23b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json:   0%|          | 0.00/711k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a794dc36ce24ef0b938a24f0861e524",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/125 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "UpdateResult(operation_id=0, status=<UpdateStatus.COMPLETED: 'completed'>)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "points = []\n",
    "\n",
    "for i, doc in enumerate(documents):\n",
    "    text = doc['question'] + ' ' + doc['text']\n",
    "    vector = models.Document(text=text, model=model_handle)\n",
    "    point = models.PointStruct(\n",
    "        id=i,\n",
    "        vector=vector,\n",
    "        payload=doc\n",
    "    )\n",
    "    points.append(point)\n",
    "\n",
    "qd_client.upsert(\n",
    "    collection_name=collection_name,\n",
    "    points=points\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "42c6665a-eb7f-426b-b6f4-9a4b779f7fcd",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = 'I just discovered the course. Can I join now?'\n",
    "\n",
    "query_points = qd_client.query_points(\n",
    "    collection_name=collection_name,\n",
    "    query=models.Document(\n",
    "        text=question,\n",
    "        model=model_handle \n",
    "    ),\n",
    "    limit=5,\n",
    "    with_payload=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "beb748e4-92ce-4ba9-8aed-773d7dc26201",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ScoredPoint(id=14, version=0, score=0.87031734, payload={'text': 'Yes, you can. You won’t be able to submit some of the homeworks, but you can still take part in the course.\\nIn order to get a certificate, you need to submit 2 out of 3 course projects and review 3 peers’ Projects by the deadline. It means that if you join the course at the end of November and manage to work on two projects, you will still be eligible for a certificate.', 'section': 'General course-related questions', 'question': 'The course has already started. Can I still join it?', 'course': 'machine-learning-zoomcamp'}, vector=None, shard_key=None, order_value=None)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_points.points[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8c8c006-ee29-44a2-ba1d-58c78b1c0d0a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
